{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CharacterTextSplitter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Document(page_content='This is the text I would like to ch'),\n",
       " Document(page_content='o chunk up. It is the example text'),\n",
       " Document(page_content='ext for this exercise')]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from langchain_text_splitters import CharacterTextSplitter\n",
    "\n",
    "text_splitter = CharacterTextSplitter(\n",
    "    separator=\"\",\n",
    "    chunk_size=35,\n",
    "    chunk_overlap=4,\n",
    ")\n",
    "\n",
    "text = \"This is the text I would like to chunk up. It is the example text for this exercise\"\n",
    "text_splitter.create_documents([text])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Document(page_content='This is\\n the text I would'),\n",
       " Document(page_content='like to chunk up.It is the example text for this exercise')]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_splitter = CharacterTextSplitter(\n",
    "    separator=\"\\n\",\n",
    "    chunk_size=35,\n",
    "    chunk_overlap=4,\n",
    ")\n",
    "\n",
    "text = \"This is\\n the text I would\\n like to chunk up.It is the example text for this exercise\"\n",
    "text_splitter.create_documents([text])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Recursively split by character"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Document(page_content='This is\\n\\n the'),\n",
       " Document(page_content='text\\n I would like'),\n",
       " Document(page_content='to chunk up. It is the example'),\n",
       " Document(page_content='text'),\n",
       " Document(page_content='for'),\n",
       " Document(page_content='this exercise')]"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from langchain_text_splitters import RecursiveCharacterTextSplitter\n",
    "\n",
    "text_splitter = RecursiveCharacterTextSplitter(\n",
    "    separators=[\"\\n\\n\", \"\\n\", \" \", \"\"],\n",
    "    chunk_size=35,\n",
    "    chunk_overlap=4,\n",
    ")\n",
    "\n",
    "text = \"This is\\n\\n the\\n\\n text\\n I would like\\n to chunk up. It is the example text\\n for\\n\\n this exercise\"\n",
    "text_splitter.create_documents([text])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Split code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Document(page_content='def hello_world():\\n    print(\"Hello, World!\")'),\n",
       " Document(page_content='# Call the function\\nhello_world()')]"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from langchain_text_splitters import (\n",
    "    Language,\n",
    "    RecursiveCharacterTextSplitter,\n",
    ")\n",
    "\n",
    "PYTHON_CODE = \"\"\"\n",
    "def hello_world():\n",
    "    print(\"Hello, World!\")\n",
    "\n",
    "# Call the function\n",
    "hello_world()\n",
    "\"\"\"\n",
    "python_splitter = RecursiveCharacterTextSplitter.from_language(\n",
    "    language=Language.PYTHON, chunk_size=50, chunk_overlap=0\n",
    ")\n",
    "python_docs = python_splitter.create_documents([PYTHON_CODE])\n",
    "python_docs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['\\nclass ', '\\ndef ', '\\n\\tdef ', '\\n\\n', '\\n', ' ', '']"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "RecursiveCharacterTextSplitter.get_separators_for_language(Language.PYTHON)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MarkdownHeaderTextSplitter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Document(page_content='Markdown[9] is a lightweight markup language for creating formatted text using a plain-text editor. John Gruber created Markdown in 2004 as a markup language that is appealing to human readers in its source code form.[9]  \\nMarkdown is widely used in blogging, instant messaging, online forums, collaborative software, documentation pages, and readme files.', metadata={'Header 1': 'Intro', 'Header 2': 'History'}),\n",
       " Document(page_content='As Markdown popularity grew rapidly, many Markdown implementations appeared, driven mostly by the need for  \\nadditional features such as tables, footnotes, definition lists,[note 1] and Markdown inside HTML blocks.  \\n#### Standardization  \\nFrom 2012, a group of people, including Jeff Atwood and John MacFarlane, launched what Atwood characterised as a standardisation effort.', metadata={'Header 1': 'Intro', 'Header 2': 'Rise and divergence'}),\n",
       " Document(page_content='Implementations of Markdown are available for over a dozen programming languages.', metadata={'Header 1': 'Intro', 'Header 2': 'Implementations'})]"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from langchain_text_splitters import MarkdownHeaderTextSplitter\n",
    "\n",
    "markdown_document = \"# Intro \\n\\n    ## History \\n\\n Markdown[9] is a lightweight markup language for creating formatted text using a plain-text editor. John Gruber created Markdown in 2004 as a markup language that is appealing to human readers in its source code form.[9] \\n\\n Markdown is widely used in blogging, instant messaging, online forums, collaborative software, documentation pages, and readme files. \\n\\n ## Rise and divergence \\n\\n As Markdown popularity grew rapidly, many Markdown implementations appeared, driven mostly by the need for \\n\\n additional features such as tables, footnotes, definition lists,[note 1] and Markdown inside HTML blocks. \\n\\n #### Standardization \\n\\n From 2012, a group of people, including Jeff Atwood and John MacFarlane, launched what Atwood characterised as a standardisation effort. \\n\\n ## Implementations \\n\\n Implementations of Markdown are available for over a dozen programming languages.\"\n",
    "\n",
    "headers_to_split_on = [\n",
    "    (\"#\", \"Header 1\"),\n",
    "    (\"##\", \"Header 2\"),\n",
    "    (\"###\", \"Header 3\"),\n",
    "]\n",
    "\n",
    "markdown_splitter = MarkdownHeaderTextSplitter(headers_to_split_on=headers_to_split_on)\n",
    "md_header_splits = markdown_splitter.split_text(markdown_document)\n",
    "md_header_splits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Document(page_content='# Intro  \\n## History  \\nMarkdown[9] is a lightweight markup language for creating formatted text using a plain-text editor. John Gruber created Markdown in 2004 as a markup language that is appealing to human readers in its source code form.[9]', metadata={'Header 1': 'Intro', 'Header 2': 'History'}),\n",
       " Document(page_content='Markdown is widely used in blogging, instant messaging, online forums, collaborative software, documentation pages, and readme files.', metadata={'Header 1': 'Intro', 'Header 2': 'History'}),\n",
       " Document(page_content='## Rise and divergence  \\nAs Markdown popularity grew rapidly, many Markdown implementations appeared, driven mostly by the need for  \\nadditional features such as tables, footnotes, definition lists,[note 1] and Markdown inside HTML blocks.', metadata={'Header 1': 'Intro', 'Header 2': 'Rise and divergence'}),\n",
       " Document(page_content='#### Standardization  \\nFrom 2012, a group of people, including Jeff Atwood and John MacFarlane, launched what Atwood characterised as a standardisation effort.', metadata={'Header 1': 'Intro', 'Header 2': 'Rise and divergence'}),\n",
       " Document(page_content='## Implementations  \\nImplementations of Markdown are available for over a dozen programming languages.', metadata={'Header 1': 'Intro', 'Header 2': 'Implementations'})]"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# MD splits\n",
    "markdown_splitter = MarkdownHeaderTextSplitter(\n",
    "    headers_to_split_on=headers_to_split_on, strip_headers=False\n",
    ")\n",
    "md_header_splits = markdown_splitter.split_text(markdown_document)\n",
    "\n",
    "# Char-level splits\n",
    "from langchain_text_splitters import RecursiveCharacterTextSplitter\n",
    "\n",
    "chunk_size = 250\n",
    "chunk_overlap = 30\n",
    "text_splitter = RecursiveCharacterTextSplitter(\n",
    "    chunk_size=chunk_size, chunk_overlap=chunk_overlap\n",
    ")\n",
    "\n",
    "# Split\n",
    "splits = text_splitter.split_documents(md_header_splits)\n",
    "splits"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Semantic Chunking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%pip install --quiet langchain_experimental"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_experimental.text_splitter import SemanticChunker\n",
    "from langchain_community.embeddings import OpenAIEmbeddings\n",
    "\n",
    "# This is a long document we can split up.\n",
    "with open(\"../../state_of_the_union.txt\") as f:\n",
    "    state_of_the_union = f.read()\n",
    "    \n",
    "text_splitter = SemanticChunker(OpenAIEmbeddings())\n",
    "\n",
    "docs = text_splitter.create_documents([state_of_the_union])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 原理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "def combine_sentences(sentences, buffer_size=1):\n",
    "    # Go through each sentence dict\n",
    "    for i in range(len(sentences)):\n",
    "\n",
    "        # Create a string that will hold the sentences which are joined\n",
    "        combined_sentence = ''\n",
    "\n",
    "        # Add sentences before the current one, based on the buffer size.\n",
    "        for j in range(i - buffer_size, i):\n",
    "            # Check if the index j is not negative (to avoid index out of range like on the first one)\n",
    "            if j >= 0:\n",
    "                # Add the sentence at index j to the combined_sentence string\n",
    "                combined_sentence += sentences[j]['sentence'] + ' '\n",
    "\n",
    "        # Add the current sentence\n",
    "        combined_sentence += sentences[i]['sentence']\n",
    "\n",
    "        # Add sentences after the current one, based on the buffer size\n",
    "        for j in range(i + 1, i + 1 + buffer_size):\n",
    "            # Check if the index j is within the range of the sentences list\n",
    "            if j < len(sentences):\n",
    "                # Add the sentence at index j to the combined_sentence string\n",
    "                combined_sentence += ' ' + sentences[j]['sentence']\n",
    "\n",
    "        # Then add the whole thing to your dict\n",
    "        # Store the combined sentence in the current sentence dict\n",
    "        sentences[i]['combined_sentence'] = combined_sentence\n",
    "\n",
    "    return sentences\n",
    "\n",
    "\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "\n",
    "\n",
    "def calculate_cosine_distances(sentences):\n",
    "    distances = []\n",
    "    for i in range(len(sentences) - 1):\n",
    "        embedding_current = sentences[i]['combined_sentence_embedding']\n",
    "        embedding_next = sentences[i + 1]['combined_sentence_embedding']\n",
    "        \n",
    "        # Calculate cosine similarity\n",
    "        similarity = cosine_similarity([embedding_current], [embedding_next])[0][0]\n",
    "        \n",
    "        # Convert to cosine distance\n",
    "        distance = 1 - similarity\n",
    "\n",
    "        # Append cosine distance to the list\n",
    "        distances.append(distance)\n",
    "\n",
    "        # Store distance in the dictionary\n",
    "        sentences[i]['distance_to_next'] = distance\n",
    "\n",
    "    # Optionally handle the last sentence\n",
    "    # sentences[-1]['distance_to_next'] = None  # or a default value\n",
    "\n",
    "    return distances, sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "317 senteneces were found\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[{'sentence': '\\n\\nWant to start a startup?', 'index': 0},\n",
       " {'sentence': 'Get funded by\\nY Combinator.', 'index': 1},\n",
       " {'sentence': 'October 2006(This essay is derived from a talk at MIT.)\\nTill recently graduating seniors had two choices: get a job or go\\nto grad school.',\n",
       "  'index': 2}]"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('./examples/mit.txt') as file:\n",
    "    essay = file.read()\n",
    "\n",
    "import re\n",
    "\n",
    "# Splitting the essay on '.', '?', and '!'\n",
    "single_sentences_list = re.split(r'(?<=[.?!])\\s+', essay)\n",
    "print (f\"{len(single_sentences_list)} senteneces were found\")\n",
    "\n",
    "sentences = [{'sentence': x, 'index' : i} for i, x in enumerate(single_sentences_list)]\n",
    "sentences[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'sentence': '\\n\\nWant to start a startup?',\n",
       "  'index': 0,\n",
       "  'combined_sentence': '\\n\\nWant to start a startup? Get funded by\\nY Combinator.'},\n",
       " {'sentence': 'Get funded by\\nY Combinator.',\n",
       "  'index': 1,\n",
       "  'combined_sentence': '\\n\\nWant to start a startup? Get funded by\\nY Combinator. October 2006(This essay is derived from a talk at MIT.)\\nTill recently graduating seniors had two choices: get a job or go\\nto grad school.'},\n",
       " {'sentence': 'October 2006(This essay is derived from a talk at MIT.)\\nTill recently graduating seniors had two choices: get a job or go\\nto grad school.',\n",
       "  'index': 2,\n",
       "  'combined_sentence': 'Get funded by\\nY Combinator. October 2006(This essay is derived from a talk at MIT.)\\nTill recently graduating seniors had two choices: get a job or go\\nto grad school. I think there will increasingly be a third option:\\nto start your own startup.'}]"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentences = combine_sentences(sentences)\n",
    "sentences[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.embeddings import OpenAIEmbeddings\n",
    "oaiembeds = OpenAIEmbeddings()\n",
    "\n",
    "embeddings = oaiembeds.embed_documents([x['combined_sentence'] for x in sentences])\n",
    "\n",
    "for i, sentence in enumerate(sentences):\n",
    "    sentence['combined_sentence_embedding'] = embeddings[i]\n",
    "\n",
    "distances, sentences = calculate_cosine_distances(sentences)\n",
    "\n",
    "distances[:3]\n",
    "\"\"\"\n",
    "[0.08081114249044896, 0.02726339916925502, 0.04722227403602797]\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "\n",
    "# We need to get the distance threshold that we'll consider an outlier\n",
    "# We'll use numpy .percentile() for this\n",
    "breakpoint_percentile_threshold = 95\n",
    "breakpoint_distance_threshold = np.percentile(distances, breakpoint_percentile_threshold) # If you want more chunks, lower the percentile cutoff\n",
    "\n",
    "# Then we'll see how many distances are actually above this one\n",
    "num_distances_above_theshold = len([x for x in distances if x > breakpoint_distance_threshold]) # The amount of distances above your threshold\n",
    "\n",
    "# Then we'll get the index of the distances that are above the threshold. This will tell us where we should split our text\n",
    "indices_above_thresh = [i for i, x in enumerate(distances) if x > breakpoint_distance_threshold]\n",
    "\n",
    "# Initialize the start index\n",
    "start_index = 0\n",
    "\n",
    "# Create a list to hold the grouped sentences\n",
    "chunks = []\n",
    "\n",
    "# Iterate through the breakpoints to slice the sentences\n",
    "for index in indices_above_thresh:\n",
    "    # The end index is the current breakpoint\n",
    "    end_index = index\n",
    "\n",
    "    # Slice the sentence_dicts from the current start index to the end index\n",
    "    group = sentences[start_index:end_index + 1]\n",
    "    combined_text = ' '.join([d['sentence'] for d in group])\n",
    "    chunks.append(combined_text)\n",
    "    \n",
    "    # Update the start index for the next group\n",
    "    start_index = index + 1\n",
    "\n",
    "# The last group, if any sentences remain\n",
    "if start_index < len(sentences):\n",
    "    combined_text = ' '.join([d['sentence'] for d in sentences[start_index:]])\n",
    "    chunks.append(combined_text)\n",
    "\n",
    "# grouped_sentences now contains the chunked sentences"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.5 (tags/v3.13.5:6cb20a2, Jun 11 2025, 16:15:46) [MSC v.1943 64 bit (AMD64)]"
  },
  "vscode": {
   "interpreter": {
    "hash": "340a546b2c4c9cf5d23eb4a2a4e78e923e0b7afe4d00162258c4442b3ee3b061"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
